//引入cheerio
import Cheerio from "cheerio";
import http from "http";
import { StringDecoder } from "string_decoder";
import SDTime from "silly-datetime"; //时间格式化处理插件
const decoder = new StringDecoder("utf-8"); //解析器，将buffer以utf-8编码方式解析
import { query } from "./mysqlQuery.js";
var reg = /https.+.html/g;
//获取整个dom对象,就是将dom结构转成可操作的jquery数据对象
// function getCurrentHtmlDom(html) {
//   //html地址或者html元素结构都型，nodejs版jquery,可以用jQuery的API来操作DOM。
//   const $ = Cheerio.load(html, {
//     decodeEntities: false, //防止.html()方法将里面的中文被编码
//   });
//   return $;
// }
//请求获取别人的html数据
function getOtherHtmlData(url) {
  //promise封装成同步代码
  return new Promise((resolve, reject) => {
    var html = "";
    http.get(url, (req, res) => {
      req.on("data", (data) => {
        html += data;
      });
      req.on("end", () => {
        const $ = Cheerio.load(html, {
          decodeEntities: false, //防止.html()方法将里面的中文被编码
        });
        resolve($); //返回这个html的dom
      });
    });
  }).catch((err) => {
    console.log(err);
  });
}
//res都是一个html页面数据对象，可直接使用类似jquery的api，因为cheerio牛逼
function getClassifyId(str) {
  let id = "";
  switch (str) {
    case "奇幻玄幻":
      id = "10001";
      break;
    case "武侠仙侠":
      id = "10002";
      break;
    case "历史军事":
      id = "10003";
      break;
    case "都市娱乐":
      id = "10004";
      break;
    case "竞技同人":
      id = "10005";
      break;
    case "科幻游戏":
      id = "10006";
      break;
    case "悬疑灵异":
      id = "10007";
      break;
    case "评论文集":
      id = "10008";
      break;
    default: //其他
      id = "10009";
  }
  return Number(id);
}
function getRandomId() {
  let num = "";
  for (let i = 0; i < 10; i++) {
    num += parseInt(Math.random() * 10);
  }
  return Number(num);
}
//html分类页码
let htmlNum = 2;
let max = 10;
//获取所有小说书本展示页链接
let bookSrcList = [];
let listSrcList = [];
//设置计时器每4分钟爬一本
let timer = null;
let timer2 = null;
//
let bookNum = 0;
let listNum = 0;
//获取所有小说展示页链接
function getBookHtmlSrcList(num) {
  return new Promise((reslove, reject) => {
    getOtherHtmlData(
      `http://book.zongheng.com/store/c0/c0/b0/u0/p${num}/v0/s1/t0/u0/i1/ALL.html`
    )
      .then((res) => {
        let arr = [];
        res(".bookinfo").each((index, ele) => {
          arr.push({
            htmlSrc: res(`.bookinfo .bookname:eq(${index})`)
              .html()
              .match(reg)[0]
              .replace(/https/, "http"),
            userName: res(`.bookinfo .bookilnk:eq(${index})`)
              .find("a:eq(0)")
              .text()
              .trim(),
          }); //因为http不能向http请求，替换链接前缀成http
        });
        reslove(arr);
      })
      .catch((err) => {
        reject(err);
      });
  });
}
//获取书本详情信息并入库
function getBookInfo(src, userName) {
  return new Promise((reslove, reject) => {
    getOtherHtmlData(src)
      .then((res) => {
        let user_id = "666666"; //当前爬取者
        let classifyId = getClassifyId(
          res(".book-info .book-label a:eq(1)").text()
        ); //书籍所属分类id
        let book_id = getRandomId(); //书id
        let authorName = userName; //作者
        let book_name = res(".book-info .book-name").text().trim(); //书名
        let book_describe = Buffer.from(
          res(".book-info .book-dec").html().trim()
        ); //书描述
        let book_words_number = res(".book-info .nums span:eq(0) i").text(); //书总字数
        let book_create_time = SDTime.format(new Date(), "YYYY-MM-DD HH:mm:ss"); //书创建并入库时间
        //获取小说主页的信息，为了获取小说目录链接
        let listSrc = res(".link-group")
          .html()
          .match(reg)[0]
          .replace(/https/, "http");
        listSrcList.push({
          book_id,
          htmlSrc: listSrc,
          bookName: book_name,
        });
        //这里就必须执行多条sql了
        query([
          {
            sql: "insert into books (user_id,classify_id,book_id,book_name,book_describe,book_words_number,book_create_time,authorName) values(?,?,?,?,?,?,?,?)",
            values: [
              user_id,
              classifyId,
              book_id,
              book_name,
              book_describe,
              book_words_number,
              book_create_time,
              authorName,
            ],
          },
        ])
          .then((res) => {
            reslove(res);
          })
          .catch((err) => {
            reject(err);
          });
      })
      .catch((err) => {
        reject(err);
      });
  });
}
function getArr5(res, book_id) {
  return new Promise((reslove, reject) => {
    //获取文章详细信息
    let obj = {
      total: res(res(".bookinfo span")[0]).text(), //章节字数
      content: Buffer.from(res(".reader_box .content").html(), "utf8"), //章节内容
      create_time: SDTime.format(new Date(), "YYYY-MM-DD HH:mm:ss"), //入库时间
      chapter_title: res(".title .title_txtbox").text().trim(), //章节标题
      book_id, //书id,
      updateTime: res(res(".bookinfo span")[1]).text().trim(), //本章创建发布时间
      chapter_id: getRandomId(), //章节id
      author: res(res(".bookinfo a")[0]).text(), //作者
      authorSrc: res(".bookinfo a")
        .eq(0)
        ["0"].attribs.href.replace(/https/, "http"), //作者中心导向
    };
    query([
      {
        sql: "insert into chapters (total,content,create_time,chapter_title,book_id,updateTime,chapter_id,author,authorSrc) values(?,?,?,?,?,?,?,?,?)",
        values: [
          obj.total,
          obj.content,
          obj.create_time,
          obj.chapter_title,
          obj.book_id,
          obj.updateTime,
          obj.chapter_id,
          obj.author,
          obj.authorSrc,
        ],
      },
    ])
      .then((res) => {
        console.log(
          `---------book_id为${book_id}的小说下的章节录入成功----------`
        );
        reslove(1);
      })
      .catch((err) => {
        reject(err)
      });
  }).catch((err) => {
    reject(err)
  });
}
//递归章节入库
function getArr4(arr) {
  console.log(arr[0]);
  let newArrPromiseList = [];
  for (let i = 0; i < arr.length; i++) {
    newArrPromiseList.push(getOtherHtmlData(arr[i].chapterSrc));
  }
  return newArrPromiseList;
}
//
//递归入库chapters
function getArr3() {
  getOtherHtmlData(listSrcList[listNum].htmlSrc)
    .then((res) => {
      console.log(`获取第${listNum+1}本小说目录所有链接成功`);
      let bookList = []; //获取所有章节标题与章节链接
      res(".col-4").each((index, ele) => {
        bookList.push({
          chapterSrc: res(ele).html().match(reg)[0].replace(/https/, "http"),
          chapterTitleName: res(ele).find("a").text(),
        });
      });
      console.log(
        `--------------开始录入${listSrcList[listNum].bookName}这本小说章节---------`
      );
      Promise.all([...getArr4(bookList)])
        .then((res) => {
          //res是这本小说所有目录链接
          let realArr = res.filter((item) => item != "err");
          let smaell = 0;
          function hbh() {
            timer = setTimeout(() => {
              console.log(
                `---------------开始第${smaell + 1}章录入--------------------`
              );
              getArr5(realArr[smaell], listSrcList[listNum].book_id).then(
                (res) => {
                  if (res == 1) {
                    console.log(
                      `--------------第${
                        smaell + 1
                      }章录入成功------------------`
                    );
                    smaell++;
                    if (smaell < realArr.length) {
                      clearTimeout(timer);
                      timer = null;
                      hbh();
                    } else {
                      clearTimeout(timer);
                      timer = null;
                      console.log(
                        `------------第${
                          listNum + 1
                        }本小说入库完成------这本小说入库完成共${smaell}章--`
                      );
                      listNum++;
                      console.log(
                        `---------------开始第${
                          listNum + 1
                        }本小说章节入库----------`
                      );
                      if (listNum < listSrcList.length) {
                        getArr3();
                      } else {
                        console.log(`共${listNum}本小说都已入库`);
                      }
                    }
                  }
                }
              ).catch((err)=>{
                console.log(err)
                clearTimeout(timer);
                timer = null;
              })
            
            }, 2000);
          }
          hbh();
        })
        .catch((err) => {
          console.log(err);
        });
    })
    .catch((err) => {
      console.log(`第本${listNum + 1}小说目录列表链接获取失败`);
      console.log(err);
    });
}
//递归入库books
function getArr2() {
  getBookInfo(bookSrcList[bookNum].htmlSrc, bookSrcList[bookNum].userName)
    .then((res) => {
      console.log(
        `-------------------第${
          bookNum + 1
        }本小说信息入库成功-------------------`
      ); //这里应该得到入库成功返回的信息
      bookNum++;
      if (bookNum < bookSrcList.length) {
        console.log(
          `-------------------开始获取第${
            bookNum + 2
          }本小说信息入库-------------------`
        );
        getArr2();
      } else {
        console.log(
          "-------------------所有小说信息全部抓取完毕-------------------"
        );
        console.log(
          "-------------------开始执行小说章节入库chapters-------------------"
        );
        getArr3();
      }
    })
    .catch((err) => {
      console.log(
        `-------------------第${bookNum}本小说信息入库失败-------------------`
      );
    });
}
//递归书库分页器抓取
function getArr() {
  getBookHtmlSrcList(htmlNum).then((res) => {
    bookSrcList.push(...res);
    htmlNum++;
    if (htmlNum <= max) {
      //等待一秒再发起请求
      timer2 = setTimeout(() => {
        clearTimeout(timer2);
        timer2 = null;
        console.log(
          `------------------已获取第${htmlNum}本超链接及作者名------------------`
        );
        getArr();
      }, 1000);
    } else {
      console.log(
        "------------------所有小说超链接已获取完毕，进行下一步信息入库-------------------"
      );
      if (bookSrcList.length > 0) {
        console.log(
          "-------------------开始第一本小说信息入库books表-------------------"
        );
        getArr2();
      } else {
        console.log(
          "-------------------未抓取到网站数据，请检查-------------------"
        );
      }
    }
  });
}
getArr();
